import scrapy

from ScrapyObject.spiders.utils.url_utils import *

'''
scrapy crawl ydav -o ydav.json
https://mypornfolder.com/
'''


class YdavSpider(scrapy.Spider):
    name = "ydav"
    # 前缀
    prefix = 'https://'
    # 中缀
    website = 'mypornfolder'
    # 后缀
    suffix = '.com/'
    allowed_domains = [website + '.com']
    start_urls = [prefix + website + suffix]

    def __init__(self):
        self.i = 0

    def parse(self, response):
        content = get_data(response)
        video_url = response.xpath("//script[@type='application/ld+json']/text()").extract()
        tags = response.xpath("//div[@class='item']//a/text()").extract()
        if len(video_url) and len(tags):
            decoded_text = video_url[0].encode('utf-8').decode('unicode_escape')
            names = re.search(r'name.*?,', decoded_text).group()
            picture_url = re.search(r'thumbnailUrl.*?,', decoded_text).group()
            video_url = re.search(r'contentUrl.*?,', decoded_text).group()
            self.i = self.i + 1
            yield get_video_item(id=self.i, tags=tags[0], url="", name=names[8:-2], pUrl=picture_url[16:-2], vUrl=video_url[14:-2])
        url_list = get_url(content)
        for url in url_list:
            if url.startswith('https'):
                yield scrapy.Request(url, callback=self.parse)
